In [76]:

    

from sklearn import datasets
import cgt
import matplotlib.pyplot as plt
import numpy as np
from cgt import nn

%matplotlib inline


    

In [87]:

    

digits = datasets.load_digits()


    

In [88]:

    

digits.data


    

    
Out[88]:





array([[  0.,   0.,   5., ...,   0.,   0.,   0.],
       [  0.,   0.,   0., ...,  10.,   0.,   0.],
       [  0.,   0.,   0., ...,  16.,   9.,   0.],
       ..., 
       [  0.,   0.,   1., ...,   6.,   0.,   0.],
       [  0.,   0.,   2., ...,  12.,   0.,   0.],
       [  0.,   0.,  10., ...,  12.,   1.,   0.]])

In [89]:

    

digits.data.shape


    

    
Out[89]:





(1797, 64)

In [90]:

    

plt.gray()
plt.matshow(digits.images[0])


    

    
Out[90]:





<matplotlib.image.AxesImage at 0x1095eed10>






    






<matplotlib.figure.Figure at 0x108fecf90>






    

In [91]:

    

len(digits.target)


    

    
Out[91]:





1797

In [92]:

    

randinds = np.random.permutation(len(digits.target))


    

In [93]:

    

# shuffle the values
from sklearn.utils import shuffle
data, targets = shuffle(digits.data, digits.target, random_state=0)


    

In [101]:

    

# scale the data
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler().fit(data)
data_scaled = scaler.transform(data)


    

In [102]:

    

from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(data_scaled, targets, test_size=0.20, random_state=0)


    

In [103]:

    

X_train.shape, y_train.shape


    

    
Out[103]:





((1437, 64), (1437,))

In [166]:

    

from cgt.distributions import categorical

def model(X, y):
    # relu(W*x + b)
    np.random.seed(0)
    h1 = nn.rectify(nn.Affine(64, 512, weight_init=nn.IIDGaussian(std=.1))(X))
    h2 = nn.rectify(nn.Affine(512, 512, weight_init=nn.IIDGaussian(std=.1))(h1))

    # softmax probabilities
    probs = nn.softmax(nn.Affine(512, 10)(h2))
    
    # our prediction is the highest probability
    ypreds = cgt.argmax(probs, axis=1)
        
    acc = cgt.cast(cgt.equal(ypreds, y), cgt.floatX).mean()
    cost = -categorical.loglik(y, probs).mean()
    
    return cost, acc


    

In [167]:

    

X = cgt.matrix(name='X', fixed_shape=(None, 64))
y = cgt.vector(name='y', dtype='i8')

cost, acc = model(X, y)


    

In [168]:

    

learning_rate = 1e-3
epochs = 100
batch_size = 64

# get all the weight parameters for our model
params = nn.get_parameters(cost)
# train via SGD, use 1e-3 as the learning rate
updates = nn.sgd(cost, params, learning_rate)


    

In [169]:

    

# Functions
trainf = cgt.function(inputs=[X,y], outputs=[], updates=updates)
cost_and_accf = cgt.function(inputs=[X,y], outputs=[cost,acc])


    

In [170]:

    

import time

for i in xrange(epochs):
    t1 = time.time()
    for srt in xrange(0, X_train.shape[0], batch_size):
        end = batch_size+srt
        trainf(X_train[srt:end], y_train[srt:end])
    elapsed = time.time() - t1
    costval, accval = cost_and_accf(X_test, y_test)
    print("Epoch {} took {}, test cost = {}, test accuracy = {}".format(i+1, elapsed, costval, accval))


    

    




Epoch 1 took 0.172019004822, test cost = 2.20501685143, test accuracy = 0.524999976158
Epoch 2 took 0.147954940796, test cost = 2.11127901077, test accuracy = 0.672222197056
Epoch 3 took 0.153834104538, test cost = 2.02107143402, test accuracy = 0.730555534363
Epoch 4 took 0.171836137772, test cost = 1.93480193615, test accuracy = 0.769444465637
Epoch 5 took 0.165919065475, test cost = 1.85273849964, test accuracy = 0.783333361149
Epoch 6 took 0.152409076691, test cost = 1.77495181561, test accuracy = 0.811111092567
Epoch 7 took 0.152605772018, test cost = 1.70139038563, test accuracy = 0.819444417953
Epoch 8 took 0.153925895691, test cost = 1.63193762302, test accuracy = 0.822222232819
Epoch 9 took 0.15299487114, test cost = 1.56644046307, test accuracy = 0.830555558205
Epoch 10 took 0.155354976654, test cost = 1.5047249794, test accuracy = 0.844444453716
Epoch 11 took 0.163406848907, test cost = 1.44660675526, test accuracy = 0.852777779102
Epoch 12 took 0.158044099808, test cost = 1.39189636707, test accuracy = 0.863888859749
Epoch 13 took 0.153048038483, test cost = 1.34040379524, test accuracy = 0.866666674614
Epoch 14 took 0.268903017044, test cost = 1.29194116592, test accuracy = 0.866666674614
Epoch 15 took 0.195513010025, test cost = 1.24632537365, test accuracy = 0.87222224474
Epoch 16 took 0.167809009552, test cost = 1.20337891579, test accuracy = 0.87777775526
Epoch 17 took 0.160308837891, test cost = 1.16293108463, test accuracy = 0.880555570126
Epoch 18 took 0.175884008408, test cost = 1.12482047081, test accuracy = 0.883333325386
Epoch 19 took 0.175163984299, test cost = 1.08889222145, test accuracy = 0.886111140251
Epoch 20 took 0.15843296051, test cost = 1.05500137806, test accuracy = 0.888888895512
Epoch 21 took 0.169517993927, test cost = 1.02301156521, test accuracy = 0.897222220898
Epoch 22 took 0.237608909607, test cost = 0.99279499054, test accuracy = 0.897222220898
Epoch 23 took 0.199369907379, test cost = 0.964232325554, test accuracy = 0.897222220898
Epoch 24 took 0.169267892838, test cost = 0.937210738659, test accuracy = 0.894444465637
Epoch 25 took 0.162634134293, test cost = 0.911626338959, test accuracy = 0.897222220898
Epoch 26 took 0.187767982483, test cost = 0.887383162975, test accuracy = 0.899999976158
Epoch 27 took 0.176151037216, test cost = 0.864389777184, test accuracy = 0.902777791023
Epoch 28 took 0.158120155334, test cost = 0.842563807964, test accuracy = 0.905555546284
Epoch 29 took 0.160732030869, test cost = 0.821827352047, test accuracy = 0.908333361149
Epoch 30 took 0.161874055862, test cost = 0.80210930109, test accuracy = 0.911111116409
Epoch 31 took 0.158382892609, test cost = 0.783342838287, test accuracy = 0.911111116409
Epoch 32 took 0.16233587265, test cost = 0.765466153622, test accuracy = 0.91388887167
Epoch 33 took 0.161313056946, test cost = 0.748422086239, test accuracy = 0.916666686535
Epoch 34 took 0.160571813583, test cost = 0.732157886028, test accuracy = 0.916666686535
Epoch 35 took 0.161388158798, test cost = 0.716625630856, test accuracy = 0.916666686535
Epoch 36 took 0.161670923233, test cost = 0.701779484749, test accuracy = 0.91388887167
Epoch 37 took 0.177587032318, test cost = 0.687576293945, test accuracy = 0.916666686535
Epoch 38 took 0.232905864716, test cost = 0.673977673054, test accuracy = 0.916666686535
Epoch 39 took 0.195561885834, test cost = 0.660947620869, test accuracy = 0.919444441795
Epoch 40 took 0.166347980499, test cost = 0.648452758789, test accuracy = 0.919444441795
Epoch 41 took 0.164025068283, test cost = 0.636462330818, test accuracy = 0.919444441795
Epoch 42 took 0.191772937775, test cost = 0.624947607517, test accuracy = 0.922222197056
Epoch 43 took 0.172203063965, test cost = 0.613881647587, test accuracy = 0.922222197056
Epoch 44 took 0.167120933533, test cost = 0.603238761425, test accuracy = 0.925000011921
Epoch 45 took 0.171378850937, test cost = 0.592996060848, test accuracy = 0.925000011921
Epoch 46 took 0.246295928955, test cost = 0.583131968975, test accuracy = 0.922222197056
Epoch 47 took 0.190277814865, test cost = 0.573626458645, test accuracy = 0.922222197056
Epoch 48 took 0.194919109344, test cost = 0.564460635185, test accuracy = 0.922222197056
Epoch 49 took 0.173241853714, test cost = 0.555616557598, test accuracy = 0.922222197056
Epoch 50 took 0.167927026749, test cost = 0.54707711935, test accuracy = 0.922222197056
Epoch 51 took 0.198837041855, test cost = 0.538827240467, test accuracy = 0.922222197056
Epoch 52 took 0.17116189003, test cost = 0.53085231781, test accuracy = 0.922222197056
Epoch 53 took 0.169182062149, test cost = 0.523139059544, test accuracy = 0.922222197056
Epoch 54 took 0.194655895233, test cost = 0.515675008297, test accuracy = 0.922222197056
Epoch 55 took 0.200953006744, test cost = 0.508448719978, test accuracy = 0.922222197056
Epoch 56 took 0.207528114319, test cost = 0.501448571682, test accuracy = 0.922222197056
Epoch 57 took 0.200693130493, test cost = 0.494664013386, test accuracy = 0.922222197056
Epoch 58 took 0.228640079498, test cost = 0.488085567951, test accuracy = 0.922222197056
Epoch 59 took 0.170794963837, test cost = 0.481703519821, test accuracy = 0.925000011921
Epoch 60 took 0.16282582283, test cost = 0.475509107113, test accuracy = 0.925000011921
Epoch 61 took 0.171529054642, test cost = 0.469494462013, test accuracy = 0.927777767181
Epoch 62 took 0.186758995056, test cost = 0.463651031256, test accuracy = 0.930555582047
Epoch 63 took 0.166273117065, test cost = 0.457971692085, test accuracy = 0.930555582047
Epoch 64 took 0.165516853333, test cost = 0.452449262142, test accuracy = 0.933333337307
Epoch 65 took 0.225577831268, test cost = 0.447077095509, test accuracy = 0.933333337307
Epoch 66 took 0.177847146988, test cost = 0.44184884429, test accuracy = 0.933333337307
Epoch 67 took 0.198083162308, test cost = 0.436759442091, test accuracy = 0.933333337307
Epoch 68 took 0.192444086075, test cost = 0.431802868843, test accuracy = 0.933333337307
Epoch 69 took 0.248876094818, test cost = 0.426974326372, test accuracy = 0.933333337307
Epoch 70 took 0.167292118073, test cost = 0.422268748283, test accuracy = 0.936111092567
Epoch 71 took 0.164566040039, test cost = 0.41768103838, test accuracy = 0.936111092567
Epoch 72 took 0.176681995392, test cost = 0.413206726313, test accuracy = 0.936111092567
Epoch 73 took 0.193591117859, test cost = 0.40884155035, test accuracy = 0.938888907433
Epoch 74 took 0.167150020599, test cost = 0.404581606388, test accuracy = 0.938888907433
Epoch 75 took 0.166520118713, test cost = 0.400422841311, test accuracy = 0.938888907433
Epoch 76 took 0.23548913002, test cost = 0.396361321211, test accuracy = 0.936111092567
Epoch 77 took 0.184077978134, test cost = 0.392393708229, test accuracy = 0.938888907433
Epoch 78 took 0.201885938644, test cost = 0.388516664505, test accuracy = 0.941666662693
Epoch 79 took 0.171756982803, test cost = 0.384726941586, test accuracy = 0.941666662693
Epoch 80 took 0.167394161224, test cost = 0.381021887064, test accuracy = 0.941666662693
Epoch 81 took 0.191834926605, test cost = 0.377398520708, test accuracy = 0.941666662693
Epoch 82 took 0.169857025146, test cost = 0.373853862286, test accuracy = 0.941666662693
Epoch 83 took 0.165904998779, test cost = 0.37038564682, test accuracy = 0.941666662693
Epoch 84 took 0.183192014694, test cost = 0.366991430521, test accuracy = 0.941666662693
Epoch 85 took 0.208754062653, test cost = 0.363668441772, test accuracy = 0.941666662693
Epoch 86 took 0.220321178436, test cost = 0.360414534807, test accuracy = 0.941666662693
Epoch 87 took 0.175114870071, test cost = 0.357226997614, test accuracy = 0.941666662693
Epoch 88 took 0.167252063751, test cost = 0.354103922844, test accuracy = 0.941666662693
Epoch 89 took 0.192193984985, test cost = 0.351043522358, test accuracy = 0.941666662693
Epoch 90 took 0.174372911453, test cost = 0.348043620586, test accuracy = 0.941666662693
Epoch 91 took 0.166671991348, test cost = 0.345102459192, test accuracy = 0.944444417953
Epoch 92 took 0.168147802353, test cost = 0.342217922211, test accuracy = 0.944444417953
Epoch 93 took 0.165232181549, test cost = 0.339388877153, test accuracy = 0.944444417953
Epoch 94 took 0.163515090942, test cost = 0.336613416672, test accuracy = 0.944444417953
Epoch 95 took 0.164340019226, test cost = 0.333889514208, test accuracy = 0.944444417953
Epoch 96 took 0.164921998978, test cost = 0.331215828657, test accuracy = 0.944444417953
Epoch 97 took 0.162013053894, test cost = 0.32859069109, test accuracy = 0.944444417953
Epoch 98 took 0.250111818314, test cost = 0.326013088226, test accuracy = 0.944444417953
Epoch 99 took 0.19277715683, test cost = 0.323481678963, test accuracy = 0.944444417953
Epoch 100 took 0.198880910873, test cost = 0.32099506259, test accuracy = 0.944444417953

In [ ]: